import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import os
import random
import sys
import logging
from urllib3.exceptions import InsecureRequestWarning
from urllib3 import disable_warnings

# 禁用 InsecureRequestWarning 警告
disable_warnings(InsecureRequestWarning)

logging.basicConfig(level=logging.INFO ,format='%(asctime)s < %(levelname)s > %(message)s', datefmt='%Y-%m-%d %H:%M:%S')

def run_script():
    name = input('请输入要搜索的关键词：')

    # 定义目标 URL
    url = f'https://github.com/search?q={name}&type=repositories'

    # 添加请求头
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }

    logging.info('正在请求目标网址')

    # 发送 GET 请求
    index = 0
    max_retries = 0
    while max_retries <= 3:
        try:
            response = requests.get(url, headers=headers, verify=False)
            # 检查请求是否成功
            if response.status_code == 200:
                logging.info(f'请求成功, 响应状态码: ({response.status_code})')
                # 解析 HTML 内容
                soup = BeautifulSoup(response.content, 'html.parser')
                with open('github.html', 'w', encoding='utf-8') as file:
                    file.write(soup.prettify())
                
                # 获取最后一个分页链接的文本
                pagination_links = soup.find_all('a', class_='Pagination__Page-sc-cp45c9-0')
                if pagination_links:
                    last_page_text = pagination_links[-2].text.strip()
                    logging.info(f'累计检索到 {last_page_text} 页数据')
                    logging.info('开始获取数据')
                else:
                    logging.warning('没有找到其他分页')
                    last_page_text = 1  # 如果没有分页链接，假设只有一页

                data = []
                for i in range(1, int(last_page_text) + 1):
                    url_list = f'{url}&p={i}'
                    print(f'正在请求第 {i} 页: {url_list}', end='\r')
                    time.sleep(0.5)

                    while True:
                        index += 1
                        response = requests.get(url_list, headers=headers, verify=False)
                        if response.status_code == 200:
                            break
                        elif response.status_code == 429:
                            print(f'{index}\请求失败，状态码: {response.status_code}，等待 5 秒后重试：', end='\r')
                            time.sleep(5)
                        elif response.status_code == 502:
                            print(f'{index}\请求失败，状态码: {response.status_code}，等待 2 秒后重试：', end='\r')
                            time.sleep(2)
                        else:
                            logging.warning(f'请求失败，状态码: {response.status_code}')
                            break
                    if response.status_code == 200:
                        soup_1 = BeautifulSoup(response.content, 'html.parser')
                        repo_items_list = soup_1.find_all('div', class_='Box-sc-g0xbh4-0 flszRz')
                        
                        for item in repo_items_list:
                            if item:
                                # 提取仓库名称
                                repo_name = item.find('a').text.strip()
                                # 提取仓库链接
                                repo_link = 'https://github.com' + item.find('a')['href']
                                # 提取仓库描述
                                repo_description = item.find('span', class_='Text__StyledText-sc-17v1xeu-0 cWNlgR search-match').text.strip() if item.find('span', class_='Text__StyledText-sc-17v1xeu-0 cWNlgR search-match') else '无描述'
                                update_time_div = item.find('div', class_='Truncate__StyledTruncate-sc-23o1d2-0 liVpTx')
                                if update_time_div:
                                    update_time_span = update_time_div.find('span', class_='Text__StyledText-sc-17v1xeu-0 hWqAbU')
                                    if update_time_span:
                                        update_time = update_time_span.text.strip()
                                    else:
                                        update_time = '无更新时间'
                                else:
                                    update_time = '无更新时间' 
                                # 提取编程语言
                                language_span = item.find('span', {'aria-label': True})
                                if language_span:
                                    language = language_span.text.strip()
                                else:
                                    language = '无语言'

                                data.append({
                                    '仓库名称': repo_name,
                                    '仓库链接': repo_link,
                                    '仓库描述': repo_description,
                                    '编程语言': language,
                                    '更新时间': update_time
                                })
                            else:
                                logging.warning('没有找到有效数据')  
                    else:
                        logging.warning(f'请求失败，状态码: {response.status_code}')
                    
                    # 添加随机延迟，防止请求过快
                    time.sleep(random.uniform(1, 5))

                logging.info('数据获取完成：')

                # 保存数据到文件
                data_df = pd.DataFrame(data)
                logging.info('数据正在保存中')
                folder_path = os.path.join('C:\\Users\\Administrator\\Desktop\\data_dir')
                os.makedirs(folder_path, exist_ok=True)  
                data_df = pd.DataFrame(data)
                file_path = os.path.join(folder_path, f'{name}.xlsx')
                data_df.to_excel(file_path, index=False)
                logging.info(f'数据保存成功：{file_path}')
                return True
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout, requests.exceptions.RequestException) as e:
            if max_retries == 3:
                logging.warning(f'请求失败，请检查网络连接或重试\n\t {e}')
                break
            else:
                max_retries += 1
                logging.warning(f'请求失败，第 {max_retries} 次重试')
                time.sleep(2)

def main():
    """主函数"""
    while True:
        run_script()
        choice = input("请选择（1.继续 2.退出）: ").strip()
        if choice == '1':
            continue  # 退出循环
        elif choice == '2':
            break  # 退出循环
        else:
            while choice!= '1' and choice!= '2':
                print("无效的选择，请重新输入。")
                choice = input("请选择（1.继续 2.退出）: ").strip()
    sys.exit # 退出驱动程序

if __name__ == '__main__':
    main()
